

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Appendix: Static TensorFlow &mdash; 简单粗暴TensorFlow 0.3 beta 文档</title>
  

  
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

  

  
        <link rel="index" title="索引"
              href="../genindex.html"/>
        <link rel="search" title="搜索" href="../search.html"/>
    <link rel="top" title="简单粗暴TensorFlow 0.3 beta 文档" href="../index.html"/>
        <link rel="prev" title="TensorFlow Extensions" href="extended.html"/> 

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../index.html" class="icon icon-home"> 简单粗暴TensorFlow
          

          
          </a>

          
            
            
              <div class="version">
                0.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">目录</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../zh/preface.html">前言</a></li>
<li class="toctree-l1"><a class="reference internal" href="../zh/installation.html">TensorFlow安装</a></li>
<li class="toctree-l1"><a class="reference internal" href="../zh/basic.html">TensorFlow基础</a></li>
<li class="toctree-l1"><a class="reference internal" href="../zh/models.html">TensorFlow模型</a></li>
<li class="toctree-l1"><a class="reference internal" href="../zh/extended.html">TensorFlow扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="../zh/static.html">附录：静态的TensorFlow</a></li>
</ul>
<p class="caption"><span class="caption-text">Contents</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="preface.html">Preface</a></li>
<li class="toctree-l1"><a class="reference internal" href="installation.html">TensorFlow Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="basic.html">TensorFlow Basic</a></li>
<li class="toctree-l1"><a class="reference internal" href="models.html">TensorFlow Models</a></li>
<li class="toctree-l1"><a class="reference internal" href="extended.html">TensorFlow Extensions</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Appendix: Static TensorFlow</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#tensorflow-1-1">TensorFlow 1+1</a></li>
<li class="toctree-l2"><a class="reference internal" href="#a-basic-example-linear-regression">A Basic Example: Linear Regression</a></li>
</ul>
</li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">简单粗暴TensorFlow</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
      <li>Appendix: Static TensorFlow</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/en/static.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="appendix-static-tensorflow">
<h1>Appendix: Static TensorFlow<a class="headerlink" href="#appendix-static-tensorflow" title="永久链接至标题">¶</a></h1>
<div class="section" id="tensorflow-1-1">
<h2>TensorFlow 1+1<a class="headerlink" href="#tensorflow-1-1" title="永久链接至标题">¶</a></h2>
<p>Essentially, TensorFlow is a symbolic computational framework (based on computational graph). Here is a “Hello World” example of computing 1+1.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="c1"># Defince a &quot;Computation Graph&quot;</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># Defince a constant Tensor</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">c</span> <span class="o">=</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>  <span class="c1"># Equal to c = tf.add(a, b)，c is a new Tensor created by Tensor a and Tesor b&#39;s add Operation</span>

<span class="n">sess</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>     <span class="c1"># Initailize a Session</span>
<span class="n">c_</span> <span class="o">=</span> <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>        <span class="c1"># Session的run() will do actually computation to the nodes (Tensor) in the Computation Graph</span>
<span class="nb">print</span><span class="p">(</span><span class="n">c_</span><span class="p">)</span>
</pre></div>
</div>
<p>Output:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="mi">2</span>
</pre></div>
</div>
<p>The program above is capable of computing 1+1 only, the following program, however, shows how to use TensorFlow to compute the sum of any two numbers through the parameter <code class="docutils literal"><span class="pre">feed_dict=</span></code> of <code class="docutils literal"><span class="pre">tf.placeholder()</span></code> and <code class="docutils literal"><span class="pre">sess.run()</span></code>:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>  <span class="c1"># Define a placeholder Tensor</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="n">c</span> <span class="o">=</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span>

<span class="n">a_</span> <span class="o">=</span> <span class="nb">input</span><span class="p">(</span><span class="s2">&quot;a = &quot;</span><span class="p">)</span>  <span class="c1"># Read an Integer from terminal and put it into a_</span>
<span class="n">b_</span> <span class="o">=</span> <span class="nb">input</span><span class="p">(</span><span class="s2">&quot;b = &quot;</span><span class="p">)</span>

<span class="n">sess</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">c_</span> <span class="o">=</span> <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="n">feed_dict</span><span class="o">=</span><span class="p">{</span><span class="n">a</span><span class="p">:</span> <span class="n">a_</span><span class="p">,</span> <span class="n">b</span><span class="p">:</span> <span class="n">b_</span><span class="p">})</span>  <span class="c1"># feed_dict will input Tensors&#39; value needed by computing c</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;a + b = </span><span class="si">%d</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">c_</span><span class="p">)</span>
</pre></div>
</div>
<p>Terminal:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="mi">3</span>
<span class="go">a + b = 5</span>
</pre></div>
</div>
<p><strong>Variable</strong> is a special type of tensor, which is built using <code class="docutils literal"><span class="pre">tf.get_variable()</span></code>. Just like variables in common progamming language, a <code class="docutils literal"><span class="pre">Variable</span></code> should be initialized before used and its value can be modified during computation in the computational graph. The following example shows how to create a <code class="docutils literal"><span class="pre">Variable</span></code>, initialize its value to 0, and increment by one.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[])</span>
<span class="n">initializer</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>   <span class="c1"># tf.assign(x, y) will return a operation “assign Tensor y&#39;s value to Tensor x”</span>
<span class="n">a_plus_1</span> <span class="o">=</span> <span class="n">a</span> <span class="o">+</span> <span class="mi">1</span>    <span class="c1"># Equal to a + tf.constant(1)</span>
<span class="n">plus_one_op</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">a_plus_1</span><span class="p">)</span>

<span class="n">sess</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">initializer</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span>
    <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">plus_one_op</span><span class="p">)</span>                   <span class="c1"># Do plus one operation to a</span>
    <span class="n">a_</span> <span class="o">=</span> <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>                        <span class="c1"># Calculate a‘s value and put the result to a_</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">a_</span><span class="p">)</span>
</pre></div>
</div>
<p>Output:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="mf">1.0</span>
<span class="mf">2.0</span>
<span class="mf">3.0</span>
<span class="mf">4.0</span>
<span class="mf">5.0</span>
</pre></div>
</div>
<p>The following code is equivalent to the code shown above. It specifies the initializer upon declaring variables and initializes all variables at once by <code class="docutils literal"><span class="pre">tf.global_variables_initializer()</span></code>, which is used more often in practical projects:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[],</span> <span class="n">initializer</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_initializer</span><span class="p">)</span>   <span class="c1"># Made initializer as a all zero initializer</span>
<span class="n">a_plus_1</span> <span class="o">=</span> <span class="n">a</span> <span class="o">+</span> <span class="mi">1</span>
<span class="n">plus_one_op</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">a_plus_1</span><span class="p">)</span>

<span class="n">sess</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">global_variables_initializer</span><span class="p">())</span> <span class="c1"># Initailize all the </span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span>
    <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">plus_one_op</span><span class="p">)</span>
    <span class="n">a_</span> <span class="o">=</span> <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">a_</span><span class="p">)</span>
</pre></div>
</div>
<p>Matrix and tensor calculation is the basic operation in scientific computation (including Machine Learning). The program shown below is to demonstrate how to calculate the product of the two matrices <img class="math" src="../_images/math/69c816dc1d0fedcc2a375b2ea3439d272cc07eeb.png" alt="\begin{bmatrix} 1 &amp; 1 &amp; 1 \\ 1 &amp; 1 &amp; 1 \end{bmatrix}"/> and <img class="math" src="../_images/math/61dbb28cf077ed543a9d7329a18cb1dcbe33b576.png" alt="\begin{bmatrix} 1 &amp; 1 \\ 1 &amp; 1 \\ 1 &amp; 1 \end{bmatrix}"/>:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="n">A</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>   <span class="c1"># tf.ones(shape) defines a all one matrix with shape</span>
<span class="n">B</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
<span class="n">C</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">)</span>

<span class="n">sess</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">C_</span> <span class="o">=</span> <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">C_</span><span class="p">)</span>
</pre></div>
</div>
<p>Output:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mf">3.</span> <span class="mf">3.</span><span class="p">]</span>
 <span class="p">[</span><span class="mf">3.</span> <span class="mf">3.</span><span class="p">]]</span>
</pre></div>
</div>
<p>Placeholders and Variables are also allowed to be vector, matrix and even higher dimentional tensor.</p>
</div>
<div class="section" id="a-basic-example-linear-regression">
<h2>A Basic Example: Linear Regression<a class="headerlink" href="#a-basic-example-linear-regression" title="永久链接至标题">¶</a></h2>
<p>Unlike previous NumPy and Eager Execution mode, TensorFlow’s Graph Execution mode uses <strong>symbolic programming</strong> for numerical operations. First, we need to abstract the computational processes into a Dataflow Graph, and represent the inputs, operations and outputs with symbolized nodes. Then, we continually send the data to the input nodes, let the data be calculated and flow along the dataflow graph, and finally reach the specific output nodes we want. The following code shows how to accomplish the same task as the code does in previous section based on TensorFlow’s symbolic programming approach, where <code class="docutils literal"><span class="pre">tf.placeholder()</span></code> can be regarded as a kind of “symbolic input node”, using <code class="docutils literal"><span class="pre">tf.get_variable()</span></code> to define the parameters of the model (the tensor of the Variable type can be assigned using <code class="docutils literal"><span class="pre">tf.assign()</span></code>), and <code class="docutils literal"><span class="pre">sess.run(output_node,</span> <span class="pre">feed_dict={input_node:</span> <span class="pre">data})</span></code> can be thought of as a process which sends data to the input node, calculates along the dataflow graph and reach the output node and eventually return values.</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="c1"># Define data flow gragh</span>
<span class="n">learning_rate_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="n">X_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">5</span><span class="p">])</span>
<span class="n">y_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">5</span><span class="p">])</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[],</span> <span class="n">initializer</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_initializer</span><span class="p">)</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[],</span> <span class="n">initializer</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_initializer</span><span class="p">)</span>

<span class="n">y_pred</span> <span class="o">=</span> <span class="n">a</span> <span class="o">*</span> <span class="n">X_</span> <span class="o">+</span> <span class="n">b</span>
<span class="n">loss</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span> <span class="o">*</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">))</span>

<span class="c1"># Back propagation, calculate gradient of variables(model parameters) manually</span>
<span class="n">grad_a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">((</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span> <span class="o">*</span> <span class="n">X_</span><span class="p">)</span>
<span class="n">grad_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span>

<span class="c1"># Gradient descent, update parameters manually</span>
<span class="n">new_a</span> <span class="o">=</span> <span class="n">a</span> <span class="o">-</span> <span class="n">learning_rate_</span> <span class="o">*</span> <span class="n">grad_a</span>
<span class="n">new_b</span> <span class="o">=</span> <span class="n">b</span> <span class="o">-</span> <span class="n">learning_rate_</span> <span class="o">*</span> <span class="n">grad_b</span>
<span class="n">update_a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">new_a</span><span class="p">)</span>
<span class="n">update_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="n">new_b</span><span class="p">)</span>

<span class="n">train_op</span> <span class="o">=</span> <span class="p">[</span><span class="n">update_a</span><span class="p">,</span> <span class="n">update_b</span><span class="p">]</span> 
<span class="c1"># End of defining of data flow gragh</span>
<span class="c1"># Attention, until now, we haven&#39;t do any actually data calculation, just defined a data flow gragh</span>

<span class="n">num_epoch</span> <span class="o">=</span> <span class="mi">10000</span>
<span class="n">learning_rate</span> <span class="o">=</span> <span class="mf">1e-3</span>
<span class="k">with</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span> <span class="k">as</span> <span class="n">sess</span><span class="p">:</span>
    <span class="c1"># Initialize variables a and b</span>
    <span class="n">tf</span><span class="o">.</span><span class="n">global_variables_initializer</span><span class="p">()</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
    <span class="c1"># Put data in the data flow gragh created above to calculate and update variables</span>
    <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_epoch</span><span class="p">):</span>
        <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">train_op</span><span class="p">,</span> <span class="n">feed_dict</span><span class="o">=</span><span class="p">{</span><span class="n">X_</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="n">y_</span><span class="p">:</span> <span class="n">y</span><span class="p">,</span> <span class="n">learning_rate_</span><span class="p">:</span> <span class="n">learning_rate</span><span class="p">})</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">([</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">]))</span>
</pre></div>
</div>
<p>In the two examples above, we manually calculated the partial derivatives of the loss function with regard to each parameter. But when both the model and the loss function become very complicated (especially in deep learning models), the workload of manual derivation is unacceptable. TensorFlow provides an <strong>automatic derivation mechanism</strong> that eliminates the hassle of manually calculating derivatives, using TensorFlow’s derivation function <code class="docutils literal"><span class="pre">tf.gradients(ys,</span> <span class="pre">xs)</span></code> to compute the partial derivatives of the loss function with regard to a and b. Thus, the two lines of code in the previous section for calculating derivatives manually,</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="c1"># Back propagation, calculate gradient of variables(model parameters) manually</span>
<span class="n">grad_a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">((</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span> <span class="o">*</span> <span class="n">X_</span><span class="p">)</span>
<span class="n">grad_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span>
</pre></div>
</div>
<p>could be replaced by</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">grad_a</span><span class="p">,</span> <span class="n">grad_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">gradients</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">])</span>
</pre></div>
</div>
<p>and the result won’t change.</p>
<p>Moreover，TensorFlow has many kinds of <strong>optimizer</strong>, which can complete derivation and gradient update together at the same time. The code in the previous section,</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="c1"># Back propagation, calculate gradient of variables(model parameters) manually</span>
<span class="n">grad_a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">((</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span> <span class="o">*</span> <span class="n">X_</span><span class="p">)</span>
<span class="n">grad_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">)</span>

<span class="c1"># Gradient descent, update parameters manually</span>
<span class="n">new_a</span> <span class="o">=</span> <span class="n">a</span> <span class="o">-</span> <span class="n">learning_rate_</span> <span class="o">*</span> <span class="n">grad_a</span>
<span class="n">new_b</span> <span class="o">=</span> <span class="n">b</span> <span class="o">-</span> <span class="n">learning_rate_</span> <span class="o">*</span> <span class="n">grad_b</span>
<span class="n">update_a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">new_a</span><span class="p">)</span>
<span class="n">update_b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="n">new_b</span><span class="p">)</span>

<span class="n">train_op</span> <span class="o">=</span> <span class="p">[</span><span class="n">update_a</span><span class="p">,</span> <span class="n">update_b</span><span class="p">]</span> 
</pre></div>
</div>
<p>could be replaced by</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">GradientDescentOptimizer</span><span class="p">(</span><span class="n">learning_rate</span><span class="o">=</span><span class="n">learning_rate_</span><span class="p">)</span>
<span class="n">grad</span> <span class="o">=</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">compute_gradients</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
<span class="n">train_op</span> <span class="o">=</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">apply_gradients</span><span class="p">(</span><span class="n">grad</span><span class="p">)</span>
</pre></div>
</div>
<p>Here, we first instantiate a gradient descent optimizer <code class="docutils literal"><span class="pre">tf.train.GradientDescentOptimizer()</span></code> in TensorFlow and set the learning rate. Then use its <code class="docutils literal"><span class="pre">compute_gradients(loss)</span></code> method to find the gradients of <code class="docutils literal"><span class="pre">loss</span></code> with regard to all variables (parameters). Finally, through the method <code class="docutils literal"><span class="pre">apply_gradients(grad)</span></code>, the variables (parameters) are updated according to the previously calculated gradients.</p>
<p>These three lines of code are equivalent to the following line of code:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">train_op</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">GradientDescentOptimizer</span><span class="p">(</span><span class="n">learning_rate</span><span class="o">=</span><span class="n">learning_rate_</span><span class="p">)</span><span class="o">.</span><span class="n">minimize</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
</pre></div>
</div>
<p>The simplified code is as follows:</p>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>

<span class="n">learning_rate_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="n">X_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">5</span><span class="p">])</span>
<span class="n">y_</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">placeholder</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[</span><span class="mi">5</span><span class="p">])</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[],</span> <span class="n">initializer</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_initializer</span><span class="p">)</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">get_variable</span><span class="p">(</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="p">[],</span> <span class="n">initializer</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_initializer</span><span class="p">)</span>

<span class="n">y_pred</span> <span class="o">=</span> <span class="n">a</span> <span class="o">*</span> <span class="n">X_</span> <span class="o">+</span> <span class="n">b</span>
<span class="n">loss</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">constant</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span> <span class="o">*</span> <span class="n">tf</span><span class="o">.</span><span class="n">reduce_sum</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_</span><span class="p">))</span>

<span class="c1"># Back propagation，calculate and update gradient of varaibles(model parameters) with TensorFlow&#39;s GradientDescentOptimier </span>
<span class="n">train_op</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">GradientDescentOptimizer</span><span class="p">(</span><span class="n">learning_rate</span><span class="o">=</span><span class="n">learning_rate_</span><span class="p">)</span><span class="o">.</span><span class="n">minimize</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>

<span class="n">num_epoch</span> <span class="o">=</span> <span class="mi">10000</span>
<span class="n">learning_rate</span> <span class="o">=</span> <span class="mf">1e-3</span>
<span class="k">with</span> <span class="n">tf</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span> <span class="k">as</span> <span class="n">sess</span><span class="p">:</span>
    <span class="n">tf</span><span class="o">.</span><span class="n">global_variables_initializer</span><span class="p">()</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
    <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_epoch</span><span class="p">):</span>
        <span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">train_op</span><span class="p">,</span> <span class="n">feed_dict</span><span class="o">=</span><span class="p">{</span><span class="n">X_</span><span class="p">:</span> <span class="n">X</span><span class="p">,</span> <span class="n">y_</span><span class="p">:</span> <span class="n">y</span><span class="p">,</span> <span class="n">learning_rate_</span><span class="p">:</span> <span class="n">learning_rate</span><span class="p">})</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">sess</span><span class="o">.</span><span class="n">run</span><span class="p">([</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">]))</span>
</pre></div>
</div>
</div>
</div>


           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
      
        <a href="extended.html" class="btn btn-neutral" title="TensorFlow Extensions" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2018, Xihan Li（雪麒）.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'0.3 beta',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>
      <script type="text/javascript" src="../_static/translations.js"></script>

  

  
  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>